#include "ctokenizer.h"

/**
 * tokenize - split file into tokens, depending on specifiend key-words
 * @param fileName
 * @param keywords
 * @param skipComments
 * @param skipWhiteSpaces
 * @param skipEndOfLines
 * @param skipTabulators
 * @return list of tokens
 */
vector<CToken> CTokenizer::tokenize(string fileName,
    const map<string, int> &keywords, bool skipComments, bool skipWhiteSpaces,
    bool skipEndOfLines, bool skipTabulators) {

  // Open file
  ifstream fh;
  fh.open(fileName.c_str(), ios_base::in);
  if (!fh.is_open()) {
    throw CFileNotExistException(getClassName(), "tokenize");
  }

  // Temporary variables
  vector<CToken> tokens;
  bool beginStr(false);
  CToken t, tmp;

  // Number of lines
  int line(1);

  // Reading tokens until endof the file
  while ((t = getToken(fh, line, keywords)).getId() != CToken::TOKEN_EOF) {

    // Skip comments, process strings, add token into list
    switch (t.getId()) {
      case CToken::TOKEN_COMMENT:
        tmp = getCommentExpression(fh);
        if (!skipComments) {
          tokens.push_back(t);
          tokens.push_back(tmp);
        }
        break;
      case CToken::TOKEN_WHITESPACE:
        if (!skipWhiteSpaces) {
          tokens.push_back(t);
        }
        break;
      case CToken::TOKEN_TABULATOR:
        if (!skipTabulators) {
          tokens.push_back(t);
        }
        break;
      case CToken::TOKEN_EOLN:
        if (!skipEndOfLines) {
          tokens.push_back(t);
        }
        line++;
        break;
      case CToken::TOKEN_STRQUOTATION:
        tokens.push_back(t);
        if ((beginStr = !beginStr)) {
          tokens.push_back(getStringExpression(fh, t.getStr()));
        }
        break;
      default:
        tokens.push_back(t);
        break;
    }
  }

  // Close file, return tokens
  fh.close();
  return tokens;
}
//---------------------------------------------------------------------------

/**
 * getToken - read token from file
 * @param fh
 * @param line
 * @param keywords
 * @return token
 */
CToken CTokenizer::getToken(ifstream &fh, int line,
    const map<string, int> &keywords) {

  // Save current position in file and read token
  int fp(fh.tellg());
  string str;
  while (true) {

    // Read character, if end of file is reached, return
    char chr;
    if (!fh.get(chr)) {
      fh.unget();
      return CToken(CToken::TOKEN_EOF);
    }

    // Add character into temporary string
    str += chr;

    // Find key-words in temporary string
    TIndex kwPos(0);
    map<string, int>::const_iterator i;
    if ((i = matchKeyword(str, kwPos, keywords)) != keywords.end()) {
      if (kwPos == 0) {
        return CToken(i->second, i->first, line);
      } else {
        fh.seekg(fp + kwPos);
        return CToken(CToken::TOKEN_EXPRESSION, str.substr(0, kwPos), line);
      }
    }
  }
}
//---------------------------------------------------------------------------

/**
 * matchKeyword - searches for key-words in string
 * @param str
 * @param pos
 * @param keywords
 * @return iterator on found key-word in map of words
 */
map<string, int>::const_iterator CTokenizer::matchKeyword(string str,
    TIndex &pos, const map<string, int> &keywords) {
  map<string, int>::const_iterator i = keywords.begin();
  for (; i != keywords.end(); i++) {
    if ((pos = str.find(i->first)) != string::npos) {
      return i;
    }
  }
  return keywords.end();
}
//---------------------------------------------------------------------------

/**
 * getCommentExpression - returns line with comment expression
 * @param fh
 * @return token
 */
CToken CTokenizer::getCommentExpression(ifstream &fh) {
  string str;
  int chr = fh.get();
  while (chr != fstream::eofbit && chr != '\n') {
    chr = fh.get();
    str += chr;
  }
  fh.unget();
  return CToken(CToken::TOKEN_EXPRESSION, str);
}
//---------------------------------------------------------------------------

/**
 * getStringExpression - returns block wtith string
 * @param fh
 * @oaram q separator determinating beginning and ending of string
 * @return token
 */
CToken CTokenizer::getStringExpression(ifstream &fh, string q) {
  string str, tmp;
  int chr = fh.get();
  while (chr != fstream::eofbit && (tmp += chr).find(q) == string::npos) {
    chr = fh.get();
    str = tmp;
  }
  fh.unget();
  return CToken(CToken::TOKEN_EXPRESSION, str);
}
//---------------------------------------------------------------------------
